package com.czl.dxpro.dx_06_taskSix;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import com.czl.dxpro.dx_01_clear.*;
import com.sun.jersey.core.impl.provider.entity.XMLJAXBElementProvider.Text;
public class dataMatchMain {

	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
		// TODO Auto-generated method stub
		Configuration conf = new Configuration();
		conf.set("mapreduce.map.memory.mb","5120");
		//不检查超时，要保证程序无死循环
		conf.set("mapred.task.timeout","0");
		conf.set("dfs.client.block.write.replace-datanode-on-failure.policy","NEVER");
		conf.set("dfs.client.block.write.replace-datanode-on-failure.enable", "TRUE");
		Job job = Job.getInstance();
		job.setJarByClass(dataMatchMain.class);
		job.setMapperClass(dataMatchMapper.class);
		//设置reduceTask 为0
		job.setNumReduceTasks(0);
		
		job.setOutputKeyClass(NullWritable.class);
		job.setOutputValueClass(Text.class);
		
		job.addCacheFile(new Path("d://data/dx/input/t_dx_basic_msg_addr.txt").toUri());
		FileInputFormat.addInputPath(job,new Path("d://data/dx/DxClear/"));
		FileOutputFormat.setOutputPath(job,new Path("d://data/dx/DxDataMatch/"));
		System.exit(job.waitForCompletion(true)?0:1);

	}

}
